# pip3 install transformers
# python3 deepseek_tokenizer.py
import transformers

# 使用DeepSeek官方模型
model_name = "deepseek-ai/deepseek-coder-6.7b-base"

tokenizer = transformers.AutoTokenizer.from_pretrained(
        model_name, trust_remote_code=True
        )

# Read content from the file
with open(r"D:\des\token.txt", "r", encoding="utf-8") as file:
    text_content = file.read()

result = tokenizer.encode(text_content)
print(result)
